from stock import Neeq, TencentStock, Snowball, My_tushare
from pymongo import MongoClient
import pymongo
import arrow
from tqdm import tqdm
import time
from multiprocessing.dummy import Process

Mongo = MongoClient('192.168.1.220',29001)
NEEQDB = Mongo['stock_neeq']
TencentDB = Mongo['stock_tencent']
SnowBallDB = Mongo['stock_snowball']
TushareDB = Mongo['tushare']
datestr = arrow.now().format('YYYYMMDD')

def neeq():
	neeq = Neeq()

	if NEEQDB['index'].find_one({'update':{'$gte':arrow.now().floor('day').datetime}}):
		symbols = [symbol.get('symbol') for symbol in NEEQDB['index'].find({'update':{'$gte':arrow.now().floor('day').datetime}},{'symbol':1})]
	else:
		coms = neeq.get_neeq_coms()
		NEEQDB['index'].insert_many(coms,ordered=False)
		symbols = [symbol.get('symbol') for symbol in coms]

	for symbol in tqdm(symbols, desc='get_neeq_detail'):	
		if not NEEQDB['detail'].find_one({'symbol':symbol,'update':{'$gte':arrow.now().floor('day').datetime}}):
			data = neeq.get_neeq_detail(symbol)
			NEEQDB['detail'].insert_one(data)

def Tencent():
	symbols = [symbol.get('symbol') for symbol in NEEQDB['index'].find({'update':{'$gte':arrow.now().floor('day').datetime}},{'symbol':1})]
	finsh = [symbol.get('symbol') for symbol in TencentDB['detail'].find({'update':{'$gte':arrow.now().floor('day').datetime}},{'symbol':1})]
	symbols = set(symbols) - set(finsh)
	ts = TencentStock()
	for symbol in tqdm(symbols,desc='get_basic_info'):
		data = ts.get_basic_info(symbol, 'nq')
		TencentDB['detail'].insert_one(data)


def snowball():
	symbols = [symbol.get('symbol') for symbol in NEEQDB['index'].find({'update':{'$gte':arrow.now().floor('day').datetime}},{'symbol':1})]
	finsh = [symbol.get('symbol') for symbol in SnowBallDB['detail'].find({'update':{'$gte':arrow.now().floor('day').datetime}},{'symbol':1})]
	detail_symbols = set(symbols) - set(finsh)

	sb = Snowball()
	for symbol in tqdm(detail_symbols, desc='get_basic_info'):
		data = sb.get_basic_info(symbol, 'oc')
		SnowBallDB['detail'].insert_one(data)
		time.sleep(0.5)

	finsh = [symbol.get('symbol') for symbol in SnowBallDB['capital_structure'].find({'update':{'$gte':arrow.now().floor('day').datetime}},{'symbol':1})]
	cs_symbols = set(symbols) - set(finsh)
	for symbol in tqdm(cs_symbols,desc='get_capital_structure'):
		data = sb.get_capital_structure(symbol, 'oc')
		try:
			if data:
				SnowBallDB['capital_structure'].insert_many(data,ordered=False)
		except pymongo.errors.BulkWriteError:
			# repeat insert
			pass
		time.sleep(0.5)

	finsh = [symbol.get('symbol') for symbol in SnowBallDB['financial_indicators'].find({'update':{'$gte':arrow.now().floor('day').datetime}},{'symbol':1})]
	fi_symbols = set(symbols) - set(finsh)
	for symbol in tqdm(fi_symbols,desc='get_financial_indicators'):
		data = sb.get_financial_indicators(symbol, 'oc')
		try:
			if data:
				SnowBallDB['financial_indicators'].insert_many(data,ordered=False)
		except pymongo.errors.BulkWriteError:
			# repeat insert
			pass
		time.sleep(0.5)

	finsh = [symbol.get('symbol') for symbol in SnowBallDB['history_stock'].find({'update':{'$gte':arrow.now().floor('day').datetime}},{'symbol':1})]
	hi_symbols = set(symbols) - set(finsh)
	for symbol in tqdm(hi_symbols,desc='get_h_stock'):
		data = sb.get_h_stock(symbol, 'oc')
		try:
			if data:
				SnowBallDB['history_stock'].insert_many(data,ordered=False)
		except pymongo.errors.BulkWriteError:
			# repeat insert
			pass
		time.sleep(0.5)

def tushare():
	mt = My_tushare()
	sb = Snowball()
	data_area = mt.get_my_area_classified()
	try:
		TushareDB['area_classified'].insert_many(data_area,ordered=False)
	except pymongo.errors.BulkWriteError:
		# repeat insert
		pass

	for each in tqdm(data_area,desc='stock like OC'):
		symbol = each.get('symbol')
		typ = 'SH' if symbol.startswith('6') else 'SZ'

		data = sb.get_basic_info(symbol, typ)
		try:
			if data:
				SnowBallDB['detail'].insert_one(data)
		except pymongo.errors.DuplicateKeyError:
			pass
		time.sleep(0.2)

		data = sb.get_capital_structure(symbol, typ)
		try:
			if data:
				SnowBallDB['capital_structure'].insert_many(data,ordered=False)
		except pymongo.errors.BulkWriteError:
			pass
		time.sleep(0.2)

		data = sb.get_financial_indicators(symbol, typ)
		try:
			if data:
				SnowBallDB['financial_indicators'].insert_many(data,ordered=False)
		except pymongo.errors.BulkWriteError:
			pass
		time.sleep(0.2)

		data = sb.get_h_stock(symbol, typ)
		try:
			if data:
				SnowBallDB['history_stock'].insert_many(data,ordered=False)
		except pymongo.errors.BulkWriteError:
			pass
		time.sleep(0.2)

	for each in tqdm(data_area,desc='get_my_h_data'):
		symbol = each.get('symbol')
		data = mt.get_my_h_data(symbol)
		try:
			if data:
				TushareDB['history_data'].insert_many(data,ordered=False)
		except pymongo.errors.BulkWriteError:
			# repeat insert
			pass


def job_1():
	neeq()
	ps = [Process(target=fun) for fun in [Tencent,snowball]]
	list(map(lambda x:x.start(),ps))
	list(map(lambda x:x.join(),ps))

def job_2():
	tushare()

# def job_1():
# 	print(1)
# 	time.sleep(2)

# def job_2():
# 	print(2)
# 	time.sleep(4)

if __name__=='__main__':
	ps = [Process(target=fun) for fun in [job_1,job_2]]
	list(map(lambda x:x.start(),ps))
	list(map(lambda x:x.join(),ps))
	# ps[0].start()
	# ps[0].join()
	print(3)
	# test()
